﻿using Common;
using Linguist.Acoustic;
using Linguist.G2p;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using Util;
using Util.Props;

namespace Linguist.Dictionary
{
    /// <summary>
    ///
    /// Creates a dictionary by reading in an ASCII-based Sphinx-3 format dictionary. Each line of the dictionary specifies
    /// the word, followed by spaces or tab, followed by the pronunciation (by way of the list of phones) of the word. Each
    /// word can have more than one pronunciations. For example, a digits dictionary will look like:
    /// <p/>
    /// <pre>
    /// ONE                  HH W AH N
    /// ONE(2)               W AH N
    /// TWO                  T UW
    /// THREE                TH R IY
    /// FOUR                 F AO R
    /// FIVE                 F AY V
    /// SIX                  S IH K S
    /// SEVEN                S EH V AH N
    /// EIGHT                EY T
    /// NINE                 N AY N
    /// ZERO                 Z IH R OW
    /// ZERO(2)              Z IY R OW
    /// OH                   OW
    /// </pre>
    /// <p/>
    /// In the above example, the words "one" and "zero" have two pronunciations each.
    /// <p/>
    /// This dictionary will read in all the words and its pronunciation(s) at startup. Therefore, if the dictionary is big,
    /// it will take longer to load and will consume more memory.
    /// </summary>
    public class FullDictionary: IDictionary
    {
        // ----------------------------------
        // configuration variables
        // ----------------------------------
        protected Boolean addSilEndingPronunciation;
        private Boolean allowMissingWords;
        private Boolean createMissingWords;
        private String wordReplacement;
        private String wordDictionaryFile;
        private String fillerDictionaryFile;
        private Boolean allocated;
        private UnitManager unitManager;
        protected String g2pModelFile;
        protected int g2pMaxPron = 0;
        protected List<String> addendaUrlList;


        private Dictionary<String, IWord> wordDictionary;
        private Dictionary<String, IWord> fillerDictionary;
        private Timer loadTimer;
        protected G2PConverter g2pDecoder;

        public FullDictionary(
                String wordDictionaryFile,
                String fillerDictionaryFile,
                List<String> addendaUrlList,
                Boolean addSilEndingPronunciation,
                String wordReplacement,
                Boolean allowMissingWords,
                Boolean createMissingWords,
                UnitManager unitManager
        ) 
        {
            

            this.wordDictionaryFile = wordDictionaryFile;
            this.fillerDictionaryFile = fillerDictionaryFile;
            this.addendaUrlList = addendaUrlList;
            this.addSilEndingPronunciation = addSilEndingPronunciation;
            this.wordReplacement = wordReplacement;
            this.allowMissingWords = allowMissingWords;
            this.createMissingWords = createMissingWords;
            this.unitManager = unitManager;
        }

        public FullDictionary(
                String wordDictionaryFile,
                String fillerDictionaryFile,
                List<String> addendaUrlList,
                Boolean addSilEndingPronunciation,
                String wordReplacement,
                Boolean allowMissingWords,
                Boolean createMissingWords,
                UnitManager unitManager, 
                String g2pModelFile,
                int g2pMaxPron
        ) 
            :this(
                    wordDictionaryFile, 
                    fillerDictionaryFile, 
                    addendaUrlList,
                    addSilEndingPronunciation,
                    wordReplacement,
                    allowMissingWords,
                    createMissingWords,
                    unitManager)
        {
            this.g2pModelFile = g2pModelFile;
            this.g2pMaxPron = g2pMaxPron;
        }

        public FullDictionary() {

        }


        /*
       /// (non-Javadoc)
        *
       /// @see edu.cmu.sphinx.util.props.Configurable#newProperties(edu.cmu.sphinx.util.props.PropertySheet)
        */
        override public void newProperties(PropertySheet ps)
        {

            wordDictionaryFile = ConfigurationManagerUtils.getResource(PROP_DICTIONARY, ps);
            fillerDictionaryFile = ConfigurationManagerUtils.getResource(PROP_FILLER_DICTIONARY, ps);
            addendaUrlList = ps.getResourceStringList(PROP_ADDENDA);
            addSilEndingPronunciation = ps.getBoolean(PROP_ADD_SIL_ENDING_PRONUNCIATION);
            wordReplacement = ps.getString(IDictionary.PROP_WORD_REPLACEMENT);
            allowMissingWords = ps.getBoolean(IDictionary.PROP_ALLOW_MISSING_WORDS);
            createMissingWords = ps.getBoolean(PROP_CREATE_MISSING_WORDS);
            unitManager = (UnitManager) ps.getComponent(PROP_UNIT_MANAGER);
            g2pModelFile = ConfigurationManagerUtils.getResource(PROP_G2P_MODEL_PATH, ps, false);
            g2pMaxPron = ps.getInt(PROP_G2P_MAX_PRONUNCIATIONS);
        }


        /* (non-Javadoc)
       /// @see edu.cmu.sphinx.linguist.dictionary.Dictionary#allocate()
        */
        override public void allocate()
        {

            if (!allocated) {
                loadTimer = TimerPool.getTimer(this, "Load Dictionary");
                loadTimer.start();
                // NOTE: "location" can be null here, in which case the
                // "wordDictionaryFile" and "fillerDictionaryFile" should
                // contain the full path to the Dictionaries.
                Trace.Write("Loading dictionary from: " + wordDictionaryFile);
                wordDictionary = loadDictionary(new FileStream(wordDictionaryFile, FileMode.Open), false);
                        
                loadCustomDictionaries(addendaUrlList);

                Trace.WriteLine("Loading filler dictionary from: " + fillerDictionaryFile);
                FileStream fillerDict = null;
                try
                {
                    if (File.Exists(fillerDictionaryFile))
                        fillerDict = new FileStream(fillerDictionaryFile, FileMode.Open);
                }
                catch(IOException e)
                {
                    Trace.WriteLine(e.Message);
                    throw e;
                }
                if(fillerDict!=null)
                    fillerDictionary = loadDictionary(fillerDict, true);

                if(g2pModelFile != null && !g2pModelFile.Equals("")) 
                {
                    g2pDecoder = new G2PConverter(g2pModelFile);
                }
                loadTimer.stop();
                Trace.WriteLine(dumpToString());
                allocated = true;
            }
        }


        /* (non-Javadoc)
       /// @see edu.cmu.sphinx.linguist.dictionary.Dictionary#deallocate()
        */
        override public void deallocate() {
            if (allocated) {
                fillerDictionary = null;
                wordDictionary = null;
                g2pDecoder = null;
                loadTimer = null;
                allocated = false;
            }
        }


        /**
        /// Loads the given sphinx3 style simple dictionary from the given InputStream. The InputStream is assumed to contain
        /// ASCII data.
         *
        /// @param inputStream  the InputStream of the dictionary
        /// @param isFillerDict true if this is a filler dictionary, false otherwise
        /// @throws java.io.IOException if there is an error reading the dictionary
         */
        protected Dictionary<String, IWord> loadDictionary(FileStream inputStream, Boolean isFillerDict)
        {
            Dictionary<String, List<IPronunciation>> dictPronunciations = new Dictionary<String, List<IPronunciation>>();
            ExtendedStreamTokenizer est = new ExtendedStreamTokenizer(inputStream,true);
            String word;
            while ((word = est.getString()) != null) 
            {
                word = removeParensFromWord(word);
                word = word.ToLower();
                List<IUnit> units = new List<IUnit>();



                List<IPronunciation> pronunciations = null;
                if (dictPronunciations != null && dictPronunciations.ContainsKey(word))
                    pronunciations =  dictPronunciations[word];
                if (pronunciations == null) 
                {
                    pronunciations = new List<IPronunciation>();
                }
            
                // Count units and add them 
                String unitText;
                while ((unitText = est.getString()) != null) 
                {
                    units.Add(getCIUnit(unitText, isFillerDict));
                }
                pronunciations.Add(new Pronunciation(units));
                // if we are adding a SIL ending duplicate
                if (!isFillerDict && addSilEndingPronunciation) 
                {
                    units.Add(UnitManager.SILENCE);
                    pronunciations.Add(new Pronunciation(units));
                }

                if (dictPronunciations.ContainsKey(word))
                    dictPronunciations[word] = pronunciations;
                else
                    dictPronunciations.Add(word, pronunciations);
            }
            inputStream.Close();
            est.close();
            return createWords(dictPronunciations, isFillerDict);
        }


        /**
        /// Converts the spelling/Pronunciations mappings in the dictionary into spelling/Word mappings.
         *
        /// @param isFillerDict if true this is a filler dictionary
         */
        protected Dictionary<String, IWord> createWords(Dictionary<String, List<IPronunciation>> 
            pronunciationList, Boolean isFillerDict) 
        {
            Dictionary <String, IWord> result = new Dictionary <String, IWord>();
            foreach (KeyValuePair<String, List<IPronunciation>> entry in  pronunciationList) 
            {
                String spelling = entry.Key;
                List<IPronunciation> pronunciations = entry.Value;
                IPronunciation[] pros = pronunciations.ToArray();

                Word word = new Word(spelling, pros, isFillerDict);
                foreach (IPronunciation pro in pros) 
                {
                    pro.setWord((IWord)word);
                }
                if (result.ContainsKey(spelling))
                    result[spelling] = word;
                else
                    result.Add(spelling, word);
            }
            return result;
        }


        /**
        /// Gets a context independent unit. There should only be one instance of any CI unit
         *
        /// @param name     the name of the unit
        /// @param isFiller if true, the unit is a filler unit
        /// @return the unit
         */
        protected Unit getCIUnit(String name, Boolean isFiller) 
        {
            return unitManager.getUnit(name, isFiller, Context.EMPTY_CONTEXT);
        }


        /**
        /// Returns a new string that is the given word but with the ending parenthesis removed.
        /// <p/>
        /// Example:
        /// <p/>
        /// <pre>
        ///  "LEAD(2)" returns "LEAD"
        ///  "LEAD" returns "LEAD"
        ///  the word to be stripped
        /// <p/>
        ///  @return the given word but with all characters from the first
        ///  open parentheses removed
         */
        protected String removeParensFromWord(String word) 
        {
            if (word[word.Length - 1] == ')') 
            {
                int index = word.LastIndexOf('(');
                if (index > 0) {
                    word = word.Substring(0, index);
                }
            }
            return word;
        }


        /**
        /// Returns a Word object based on the spelling and its classification. The behavior of this method is affected by
        /// the properties wordReplacement, allowMissingWords, and createMissingWords.
         *
        /// @param text the spelling of the word of interest.
        /// @return a Word object
        /// @see edu.cmu.sphinx.linguist.dictionary.Word
         */
        override public IWord getWord(String text) 
        {
            text = text.ToLower();
            IWord word = lookupWord(text);
            if (word == null) 
            {
                Trace.WriteLine("The dictionary is missing a phonetic transcription for the word '" + text + "'");
                if (wordReplacement != null) 
                {
                    word = lookupWord(wordReplacement);
                    Trace.WriteLine("Replacing " + text + " with " +
                            wordReplacement);
                    if (word == null) 
                    {
                        Trace.WriteLine("Replacement word " + wordReplacement
                                + " not found!");
                    }
                } 
                else if (allowMissingWords) 
                {
                    if (createMissingWords) 
                    {
                        if (g2pModelFile != null && !g2pModelFile.Equals("")) 
                        {
                            Trace.WriteLine("Generating phonetic transcription(s) for the word '" + text + "' using g2p model");
                            List<G2p.Path> paths = g2pDecoder.phoneticize(text, g2pMaxPron);
                            List<Pronunciation> pronunciations = new List<Pronunciation>();
                            foreach(G2p.Path p in paths) 
                            {
                                int unitCount = p.getPath().Count;
                                List<IUnit> units = new List<IUnit>(unitCount);
                                foreach(String token in p.getPath()) 
                                {
                                    units.Add(getCIUnit(token, false));
                                }
                                if (units.Count == 0) 
                                {
                            	    units.Add(UnitManager.SILENCE);
                                }
                                pronunciations.Add(new Pronunciation(units));
                                if (addSilEndingPronunciation) 
                                {
                                    units.Add(UnitManager.SILENCE);
                                    pronunciations.Add(new Pronunciation(units));
                                }
                            }
                            IPronunciation[] pronunciationsArray = pronunciations.ToArray();
                            word = new Word(text, pronunciationsArray, false);
                            foreach (IPronunciation pronunciation in pronunciationsArray) 
                            {
                                pronunciation.setWord(word);
                            }
                            if (wordDictionary.ContainsKey(text))
                                wordDictionary[text] = word;
                            else
                                wordDictionary.Add(text, word);
                        } 
                        else 
                        {
                            word = new Word(text, null, false);
                            if (wordDictionary.ContainsKey(text))
                                wordDictionary[text] = word;
                            else
                                wordDictionary.Add(text, word);
                            return null;
                        }
                    } 
                    else 
                    {
                        return null;
                    }
                } 
                else 
                {
                    return null;
                }
            }
            return word;
        }


        /**
        /// Lookups up a word
         *
        /// @param spelling the spelling of the word
        /// @return the word or null
         */
        private IWord lookupWord(String spelling) 
        {
            IWord word= null;
            if(wordDictionary.ContainsKey(spelling))
                word = wordDictionary[spelling];
            if (word == null) 
            {
                if (fillerDictionary != null && fillerDictionary.ContainsKey(spelling))
                    word = fillerDictionary[spelling];
            }
            return word;
        }


        /**
        /// Returns the sentence start word.
         *
        /// @return the sentence start word
         */
        override public IWord getSentenceStartWord() 
        {
            return getWord(SENTENCE_START_SPELLING);
        }


        /**
        /// Returns the sentence end word.
         *
        /// @return the sentence end word
         */
        override public IWord getSentenceEndWord() 
        {
            return getWord(SENTENCE_END_SPELLING);
        }


        /**
        /// Returns the silence word.
         *
        /// @return the silence word
         */
        override public IWord getSilenceWord() 
        {
            return getWord(SILENCE_SPELLING);
        }


        /**
        /// Returns the set of all possible word classifications for this dictionary.
         *
        /// @return the set of all possible word classifications
         */
        override public IWordClassification[] getPossibleWordClassifications() 
        {
            return null;
        }


        /**
        /// Get the word dictionary file
         *
        /// @return the URL of the word dictionary file
         */
        public String getWordDictionaryFile() 
        {
            return wordDictionaryFile;
        }


        /**
        /// Get the filler dictionary file
         *
        /// @return the URL of the filler dictionary file
         */
        public String getFillerDictionaryFile() 
        {
            return fillerDictionaryFile;
        }


        /**
        /// Returns a string representation of this FullDictionary in alphabetical order.
         *
        /// @return a string representation of this FullDictionary
         */
        override
        public String ToString() 
        {
            return base.ToString() + "numWords=" + wordDictionary.Count.ToString() + " dictLlocation=" + getWordDictionaryFile();
        }


        private String dumpToString() 
        {
            SortedDictionary<String, IWord> sortedDict = new SortedDictionary<string, IWord>(wordDictionary);
            if(fillerDictionary!=null)
                sortedDict.Concat(fillerDictionary);
            StringBuilder result = new StringBuilder();
            //foreach(KeyValuePair<String, Word> iterator in fillerDictionary)
            //    sorted.Add(iterator);

            foreach (String text in sortedDict.Keys) 
            {
                IWord word = getWord(text);
                IPronunciation[] pronunciations = word.getPronunciations(null);
                result.Append(word).Append('\n');
                foreach (IPronunciation pronunciation in pronunciations) 
                {
                    result.Append("   ").Append(pronunciation.ToString()).Append('\n');
                }
            }
            return result.ToString();
        }


        /**
        /// Gets the set of all filler words in the dictionary
         *
        /// @return an array (possibly empty) of all filler words
         */
        override public IWord[] getFillerWords() 
        {
            return fillerDictionary.Values.ToArray();
        }
    
        /**
        /// Loads the dictionary with a list of URLs to custom dictionary resources
         *
        /// @param addenda the list of custom dictionary URLs to be loaded
        /// @throws IOException if there is an error reading the resource URL
         */
        private void loadCustomDictionaries(List<String> addenda) 
        {
            if (addenda != null) {
                foreach (String addendumUrl in addenda) 
                {
                    Trace.WriteLine("Loading addendum dictionary from: " + addendumUrl);
                    loadDictionary(new FileStream(addendumUrl, FileMode.Open), false);
                }
            }
        }

    }
}
